{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```powershell\n",
    "conda activate fastchat\n",
    "python -m fastchat.serve.controller\n",
    "python -m fastchat.serve.model_worker --model-names \"gpt-3.5-turbo,text-davinci-003,text-embedding-ada-002\" --model-path lmsys/vicuna-7b-v1.3\n",
    "python -m fastchat.serve.openai_api_server --host localhost --port 8000\n",
    "pip install ipywidgets\n",
    "pip install faiss-gpu\n",
    "pip install faiss-cpu\n",
    "pip install --use-pep517 faiss-gpu\n",
    "\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Summaries Of Short Text\n",
    "\n",
    "from langchain.llms import OpenAI\n",
    "from langchain import PromptTemplate\n",
    "\n",
    "openai_api_key = \"EMPTY\"\n",
    "openai_api_base=\"http://localhost:8000/v1\"\n",
    "\n",
    "llm = OpenAI(temperature=0, model_name = 'gpt-3.5-turbo',openai_api_base=openai_api_base, openai_api_key=openai_api_key) # 初始化LLM模型\n",
    "\n",
    "# 创建模板\n",
    "template = \"\"\"\n",
    "%INSTRUCTIONS:\n",
    "Please summarize the following piece of text.\n",
    "Respond in a manner that a 5 year old would understand.\n",
    "\n",
    "%TEXT:\n",
    "{text}\n",
    "\"\"\"\n",
    "\n",
    "# 创建一个 Lang Chain Prompt 模板，稍后可以插入值\n",
    "prompt = PromptTemplate(\n",
    "    input_variables=[\"text\"],\n",
    "    template=template,\n",
    ")\n",
    "confusing_text = \"\"\"\n",
    "For the next 130 years, debate raged.\n",
    "Some scientists called Prototaxites a lichen, others a fungus, and still others clung to the notion that it was some kind of tree.\n",
    "“The problem is that when you look up close at the anatomy, it’s evocative of a lot of different things, but it’s diagnostic of nothing,” says Boyce, an associate professor in geophysical sciences and the Committee on Evolutionary Biology.\n",
    "“And it’s so damn big that when whenever someone says it’s something, everyone else’s hackles get up: ‘How could you have a lichen 20 feet tall?’”\n",
    "\"\"\"\n",
    "\n",
    "print (\"------- Prompt Begin -------\")\n",
    "# 打印模板内容\n",
    "final_prompt = prompt.format(text=confusing_text)\n",
    "print(final_prompt)\n",
    "\n",
    "print (\"------- Prompt End -------\")\n",
    "\n",
    "output = llm(final_prompt)\n",
    "print (output)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f0eecc29e024462898282aadea99577",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The model 'BaichuanForCausalLM' is not supported for text2text-generation. Supported models are ['BartForConditionalGeneration', 'BigBirdPegasusForConditionalGeneration', 'BlenderbotForConditionalGeneration', 'BlenderbotSmallForConditionalGeneration', 'EncoderDecoderModel', 'FSMTForConditionalGeneration', 'GPTSanJapaneseForConditionalGeneration', 'LEDForConditionalGeneration', 'LongT5ForConditionalGeneration', 'M2M100ForConditionalGeneration', 'MarianMTModel', 'MBartForConditionalGeneration', 'MT5ForConditionalGeneration', 'MvpForConditionalGeneration', 'NllbMoeForConditionalGeneration', 'PegasusForConditionalGeneration', 'PegasusXForConditionalGeneration', 'PLBartForConditionalGeneration', 'ProphetNetForConditionalGeneration', 'SwitchTransformersForConditionalGeneration', 'T5ForConditionalGeneration', 'UMT5ForConditionalGeneration', 'XLMProphetNetForConditionalGeneration'].\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. Members of Congress and the Cabinet. Justices of the Supreme Court. My fellow Americans.  \n",
      "\n",
      "Last year COVID-19 kept us apart. This year we are finally together again. \n",
      "\n",
      "Tonight, we meet as Democrats Republicans \n",
      "There are 9427 tokens in your file\n",
      "You now have 9 docs intead of 1 piece of text\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32md:\\dev\\go-ai\\App\\code\\langchain\\demo_langchain.ipynb Cell 3\u001b[0m line \u001b[0;36m5\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/dev/go-ai/App/code/langchain/demo_langchain.ipynb#W2sZmlsZQ%3D%3D?line=51'>52</a>\u001b[0m chain \u001b[39m=\u001b[39m load_summarize_chain(llm\u001b[39m=\u001b[39mllm, chain_type\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mmap_reduce\u001b[39m\u001b[39m'\u001b[39m) \u001b[39m# verbose=True 展示运行日志\u001b[39;00m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/dev/go-ai/App/code/langchain/demo_langchain.ipynb#W2sZmlsZQ%3D%3D?line=53'>54</a>\u001b[0m \u001b[39m# Use it. This will run through the many documents, summarize the chunks, then get a summary of the summary.\u001b[39;00m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/dev/go-ai/App/code/langchain/demo_langchain.ipynb#W2sZmlsZQ%3D%3D?line=54'>55</a>\u001b[0m \u001b[39m# 典型的map reduce的思路去解决问题，将文章拆分成多个部分，再将多个部分分别进行 summarize，最后再进行 合并，对 summarys 进行 summary\u001b[39;00m\n\u001b[1;32m---> <a href='vscode-notebook-cell:/d%3A/dev/go-ai/App/code/langchain/demo_langchain.ipynb#W2sZmlsZQ%3D%3D?line=55'>56</a>\u001b[0m output \u001b[39m=\u001b[39m chain\u001b[39m.\u001b[39;49mrun(docs)\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/dev/go-ai/App/code/langchain/demo_langchain.ipynb#W2sZmlsZQ%3D%3D?line=56'>57</a>\u001b[0m \u001b[39mprint\u001b[39m (output)\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\chains\\base.py:487\u001b[0m, in \u001b[0;36mChain.run\u001b[1;34m(self, callbacks, tags, metadata, *args, **kwargs)\u001b[0m\n\u001b[0;32m    485\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(args) \u001b[39m!=\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[0;32m    486\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39m`run` supports only one positional argument.\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m--> 487\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m(args[\u001b[39m0\u001b[39;49m], callbacks\u001b[39m=\u001b[39;49mcallbacks, tags\u001b[39m=\u001b[39;49mtags, metadata\u001b[39m=\u001b[39;49mmetadata)[\n\u001b[0;32m    488\u001b[0m         _output_key\n\u001b[0;32m    489\u001b[0m     ]\n\u001b[0;32m    491\u001b[0m \u001b[39mif\u001b[39;00m kwargs \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m args:\n\u001b[0;32m    492\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m(kwargs, callbacks\u001b[39m=\u001b[39mcallbacks, tags\u001b[39m=\u001b[39mtags, metadata\u001b[39m=\u001b[39mmetadata)[\n\u001b[0;32m    493\u001b[0m         _output_key\n\u001b[0;32m    494\u001b[0m     ]\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\chains\\base.py:292\u001b[0m, in \u001b[0;36mChain.__call__\u001b[1;34m(self, inputs, return_only_outputs, callbacks, tags, metadata, run_name, include_run_info)\u001b[0m\n\u001b[0;32m    290\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m    291\u001b[0m     run_manager\u001b[39m.\u001b[39mon_chain_error(e)\n\u001b[1;32m--> 292\u001b[0m     \u001b[39mraise\u001b[39;00m e\n\u001b[0;32m    293\u001b[0m run_manager\u001b[39m.\u001b[39mon_chain_end(outputs)\n\u001b[0;32m    294\u001b[0m final_outputs: Dict[\u001b[39mstr\u001b[39m, Any] \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mprep_outputs(\n\u001b[0;32m    295\u001b[0m     inputs, outputs, return_only_outputs\n\u001b[0;32m    296\u001b[0m )\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\chains\\base.py:286\u001b[0m, in \u001b[0;36mChain.__call__\u001b[1;34m(self, inputs, return_only_outputs, callbacks, tags, metadata, run_name, include_run_info)\u001b[0m\n\u001b[0;32m    279\u001b[0m run_manager \u001b[39m=\u001b[39m callback_manager\u001b[39m.\u001b[39mon_chain_start(\n\u001b[0;32m    280\u001b[0m     dumpd(\u001b[39mself\u001b[39m),\n\u001b[0;32m    281\u001b[0m     inputs,\n\u001b[0;32m    282\u001b[0m     name\u001b[39m=\u001b[39mrun_name,\n\u001b[0;32m    283\u001b[0m )\n\u001b[0;32m    284\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m    285\u001b[0m     outputs \u001b[39m=\u001b[39m (\n\u001b[1;32m--> 286\u001b[0m         \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_call(inputs, run_manager\u001b[39m=\u001b[39;49mrun_manager)\n\u001b[0;32m    287\u001b[0m         \u001b[39mif\u001b[39;00m new_arg_supported\n\u001b[0;32m    288\u001b[0m         \u001b[39melse\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_call(inputs)\n\u001b[0;32m    289\u001b[0m     )\n\u001b[0;32m    290\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m    291\u001b[0m     run_manager\u001b[39m.\u001b[39mon_chain_error(e)\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\chains\\combine_documents\\base.py:105\u001b[0m, in \u001b[0;36mBaseCombineDocumentsChain._call\u001b[1;34m(self, inputs, run_manager)\u001b[0m\n\u001b[0;32m    103\u001b[0m \u001b[39m# Other keys are assumed to be needed for LLM prediction\u001b[39;00m\n\u001b[0;32m    104\u001b[0m other_keys \u001b[39m=\u001b[39m {k: v \u001b[39mfor\u001b[39;00m k, v \u001b[39min\u001b[39;00m inputs\u001b[39m.\u001b[39mitems() \u001b[39mif\u001b[39;00m k \u001b[39m!=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39minput_key}\n\u001b[1;32m--> 105\u001b[0m output, extra_return_dict \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcombine_docs(\n\u001b[0;32m    106\u001b[0m     docs, callbacks\u001b[39m=\u001b[39m_run_manager\u001b[39m.\u001b[39mget_child(), \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mother_keys\n\u001b[0;32m    107\u001b[0m )\n\u001b[0;32m    108\u001b[0m extra_return_dict[\u001b[39mself\u001b[39m\u001b[39m.\u001b[39moutput_key] \u001b[39m=\u001b[39m output\n\u001b[0;32m    109\u001b[0m \u001b[39mreturn\u001b[39;00m extra_return_dict\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\chains\\combine_documents\\map_reduce.py:209\u001b[0m, in \u001b[0;36mMapReduceDocumentsChain.combine_docs\u001b[1;34m(self, docs, token_max, callbacks, **kwargs)\u001b[0m\n\u001b[0;32m    197\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mcombine_docs\u001b[39m(\n\u001b[0;32m    198\u001b[0m     \u001b[39mself\u001b[39m,\n\u001b[0;32m    199\u001b[0m     docs: List[Document],\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    202\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs: Any,\n\u001b[0;32m    203\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m Tuple[\u001b[39mstr\u001b[39m, \u001b[39mdict\u001b[39m]:\n\u001b[0;32m    204\u001b[0m \u001b[39m    \u001b[39m\u001b[39m\"\"\"Combine documents in a map reduce manner.\u001b[39;00m\n\u001b[0;32m    205\u001b[0m \n\u001b[0;32m    206\u001b[0m \u001b[39m    Combine by mapping first chain over all documents, then reducing the results.\u001b[39;00m\n\u001b[0;32m    207\u001b[0m \u001b[39m    This reducing can be done recursively if needed (if there are many documents).\u001b[39;00m\n\u001b[0;32m    208\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 209\u001b[0m     map_results \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mllm_chain\u001b[39m.\u001b[39;49mapply(\n\u001b[0;32m    210\u001b[0m         \u001b[39m# FYI - this is parallelized and so it is fast.\u001b[39;49;00m\n\u001b[0;32m    211\u001b[0m         [{\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdocument_variable_name: d\u001b[39m.\u001b[39;49mpage_content, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs} \u001b[39mfor\u001b[39;49;00m d \u001b[39min\u001b[39;49;00m docs],\n\u001b[0;32m    212\u001b[0m         callbacks\u001b[39m=\u001b[39;49mcallbacks,\n\u001b[0;32m    213\u001b[0m     )\n\u001b[0;32m    214\u001b[0m     question_result_key \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mllm_chain\u001b[39m.\u001b[39moutput_key\n\u001b[0;32m    215\u001b[0m     result_docs \u001b[39m=\u001b[39m [\n\u001b[0;32m    216\u001b[0m         Document(page_content\u001b[39m=\u001b[39mr[question_result_key], metadata\u001b[39m=\u001b[39mdocs[i]\u001b[39m.\u001b[39mmetadata)\n\u001b[0;32m    217\u001b[0m         \u001b[39m# This uses metadata from the docs, and the textual results from `results`\u001b[39;00m\n\u001b[0;32m    218\u001b[0m         \u001b[39mfor\u001b[39;00m i, r \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(map_results)\n\u001b[0;32m    219\u001b[0m     ]\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\chains\\llm.py:191\u001b[0m, in \u001b[0;36mLLMChain.apply\u001b[1;34m(self, input_list, callbacks)\u001b[0m\n\u001b[0;32m    189\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m    190\u001b[0m     run_manager\u001b[39m.\u001b[39mon_chain_error(e)\n\u001b[1;32m--> 191\u001b[0m     \u001b[39mraise\u001b[39;00m e\n\u001b[0;32m    192\u001b[0m outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcreate_outputs(response)\n\u001b[0;32m    193\u001b[0m run_manager\u001b[39m.\u001b[39mon_chain_end({\u001b[39m\"\u001b[39m\u001b[39moutputs\u001b[39m\u001b[39m\"\u001b[39m: outputs})\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\chains\\llm.py:188\u001b[0m, in \u001b[0;36mLLMChain.apply\u001b[1;34m(self, input_list, callbacks)\u001b[0m\n\u001b[0;32m    183\u001b[0m run_manager \u001b[39m=\u001b[39m callback_manager\u001b[39m.\u001b[39mon_chain_start(\n\u001b[0;32m    184\u001b[0m     dumpd(\u001b[39mself\u001b[39m),\n\u001b[0;32m    185\u001b[0m     {\u001b[39m\"\u001b[39m\u001b[39minput_list\u001b[39m\u001b[39m\"\u001b[39m: input_list},\n\u001b[0;32m    186\u001b[0m )\n\u001b[0;32m    187\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> 188\u001b[0m     response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mgenerate(input_list, run_manager\u001b[39m=\u001b[39;49mrun_manager)\n\u001b[0;32m    189\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m    190\u001b[0m     run_manager\u001b[39m.\u001b[39mon_chain_error(e)\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\chains\\llm.py:103\u001b[0m, in \u001b[0;36mLLMChain.generate\u001b[1;34m(self, input_list, run_manager)\u001b[0m\n\u001b[0;32m    101\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Generate LLM result from inputs.\"\"\"\u001b[39;00m\n\u001b[0;32m    102\u001b[0m prompts, stop \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mprep_prompts(input_list, run_manager\u001b[39m=\u001b[39mrun_manager)\n\u001b[1;32m--> 103\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mllm\u001b[39m.\u001b[39mgenerate_prompt(\n\u001b[0;32m    104\u001b[0m     prompts,\n\u001b[0;32m    105\u001b[0m     stop,\n\u001b[0;32m    106\u001b[0m     callbacks\u001b[39m=\u001b[39mrun_manager\u001b[39m.\u001b[39mget_child() \u001b[39mif\u001b[39;00m run_manager \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m,\n\u001b[0;32m    107\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mllm_kwargs,\n\u001b[0;32m    108\u001b[0m )\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\llms\\base.py:492\u001b[0m, in \u001b[0;36mBaseLLM.generate_prompt\u001b[1;34m(self, prompts, stop, callbacks, **kwargs)\u001b[0m\n\u001b[0;32m    484\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mgenerate_prompt\u001b[39m(\n\u001b[0;32m    485\u001b[0m     \u001b[39mself\u001b[39m,\n\u001b[0;32m    486\u001b[0m     prompts: List[PromptValue],\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    489\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs: Any,\n\u001b[0;32m    490\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m LLMResult:\n\u001b[0;32m    491\u001b[0m     prompt_strings \u001b[39m=\u001b[39m [p\u001b[39m.\u001b[39mto_string() \u001b[39mfor\u001b[39;00m p \u001b[39min\u001b[39;00m prompts]\n\u001b[1;32m--> 492\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mgenerate(prompt_strings, stop\u001b[39m=\u001b[39mstop, callbacks\u001b[39m=\u001b[39mcallbacks, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\llms\\base.py:627\u001b[0m, in \u001b[0;36mBaseLLM.generate\u001b[1;34m(self, prompts, stop, callbacks, tags, metadata, **kwargs)\u001b[0m\n\u001b[0;32m    618\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\n\u001b[0;32m    619\u001b[0m             \u001b[39m\"\u001b[39m\u001b[39mAsked to cache, but no cache found at `langchain.cache`.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[0;32m    620\u001b[0m         )\n\u001b[0;32m    621\u001b[0m     run_managers \u001b[39m=\u001b[39m [\n\u001b[0;32m    622\u001b[0m         callback_manager\u001b[39m.\u001b[39mon_llm_start(\n\u001b[0;32m    623\u001b[0m             dumpd(\u001b[39mself\u001b[39m), [prompt], invocation_params\u001b[39m=\u001b[39mparams, options\u001b[39m=\u001b[39moptions\n\u001b[0;32m    624\u001b[0m         )[\u001b[39m0\u001b[39m]\n\u001b[0;32m    625\u001b[0m         \u001b[39mfor\u001b[39;00m callback_manager, prompt \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(callback_managers, prompts)\n\u001b[0;32m    626\u001b[0m     ]\n\u001b[1;32m--> 627\u001b[0m     output \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_generate_helper(\n\u001b[0;32m    628\u001b[0m         prompts, stop, run_managers, \u001b[39mbool\u001b[39m(new_arg_supported), \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs\n\u001b[0;32m    629\u001b[0m     )\n\u001b[0;32m    630\u001b[0m     \u001b[39mreturn\u001b[39;00m output\n\u001b[0;32m    631\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(missing_prompts) \u001b[39m>\u001b[39m \u001b[39m0\u001b[39m:\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\llms\\base.py:529\u001b[0m, in \u001b[0;36mBaseLLM._generate_helper\u001b[1;34m(self, prompts, stop, run_managers, new_arg_supported, **kwargs)\u001b[0m\n\u001b[0;32m    527\u001b[0m     \u001b[39mfor\u001b[39;00m run_manager \u001b[39min\u001b[39;00m run_managers:\n\u001b[0;32m    528\u001b[0m         run_manager\u001b[39m.\u001b[39mon_llm_error(e)\n\u001b[1;32m--> 529\u001b[0m     \u001b[39mraise\u001b[39;00m e\n\u001b[0;32m    530\u001b[0m flattened_outputs \u001b[39m=\u001b[39m output\u001b[39m.\u001b[39mflatten()\n\u001b[0;32m    531\u001b[0m \u001b[39mfor\u001b[39;00m manager, flattened_output \u001b[39min\u001b[39;00m \u001b[39mzip\u001b[39m(run_managers, flattened_outputs):\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\llms\\base.py:516\u001b[0m, in \u001b[0;36mBaseLLM._generate_helper\u001b[1;34m(self, prompts, stop, run_managers, new_arg_supported, **kwargs)\u001b[0m\n\u001b[0;32m    506\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_generate_helper\u001b[39m(\n\u001b[0;32m    507\u001b[0m     \u001b[39mself\u001b[39m,\n\u001b[0;32m    508\u001b[0m     prompts: List[\u001b[39mstr\u001b[39m],\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    512\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs: Any,\n\u001b[0;32m    513\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m LLMResult:\n\u001b[0;32m    514\u001b[0m     \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m    515\u001b[0m         output \u001b[39m=\u001b[39m (\n\u001b[1;32m--> 516\u001b[0m             \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_generate(\n\u001b[0;32m    517\u001b[0m                 prompts,\n\u001b[0;32m    518\u001b[0m                 stop\u001b[39m=\u001b[39mstop,\n\u001b[0;32m    519\u001b[0m                 \u001b[39m# TODO: support multiple run managers\u001b[39;00m\n\u001b[0;32m    520\u001b[0m                 run_manager\u001b[39m=\u001b[39mrun_managers[\u001b[39m0\u001b[39m] \u001b[39mif\u001b[39;00m run_managers \u001b[39melse\u001b[39;00m \u001b[39mNone\u001b[39;00m,\n\u001b[0;32m    521\u001b[0m                 \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs,\n\u001b[0;32m    522\u001b[0m             )\n\u001b[0;32m    523\u001b[0m             \u001b[39mif\u001b[39;00m new_arg_supported\n\u001b[0;32m    524\u001b[0m             \u001b[39melse\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_generate(prompts, stop\u001b[39m=\u001b[39mstop)\n\u001b[0;32m    525\u001b[0m         )\n\u001b[0;32m    526\u001b[0m     \u001b[39mexcept\u001b[39;00m \u001b[39mBaseException\u001b[39;00m \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m    527\u001b[0m         \u001b[39mfor\u001b[39;00m run_manager \u001b[39min\u001b[39;00m run_managers:\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\llms\\base.py:1006\u001b[0m, in \u001b[0;36mLLM._generate\u001b[1;34m(self, prompts, stop, run_manager, **kwargs)\u001b[0m\n\u001b[0;32m   1003\u001b[0m new_arg_supported \u001b[39m=\u001b[39m inspect\u001b[39m.\u001b[39msignature(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_call)\u001b[39m.\u001b[39mparameters\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mrun_manager\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m   1004\u001b[0m \u001b[39mfor\u001b[39;00m prompt \u001b[39min\u001b[39;00m prompts:\n\u001b[0;32m   1005\u001b[0m     text \u001b[39m=\u001b[39m (\n\u001b[1;32m-> 1006\u001b[0m         \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_call(prompt, stop\u001b[39m=\u001b[39mstop, run_manager\u001b[39m=\u001b[39mrun_manager, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m   1007\u001b[0m         \u001b[39mif\u001b[39;00m new_arg_supported\n\u001b[0;32m   1008\u001b[0m         \u001b[39melse\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_call(prompt, stop\u001b[39m=\u001b[39mstop, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m   1009\u001b[0m     )\n\u001b[0;32m   1010\u001b[0m     generations\u001b[39m.\u001b[39mappend([Generation(text\u001b[39m=\u001b[39mtext)])\n\u001b[0;32m   1011\u001b[0m \u001b[39mreturn\u001b[39;00m LLMResult(generations\u001b[39m=\u001b[39mgenerations)\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\langchain\\llms\\huggingface_pipeline.py:167\u001b[0m, in \u001b[0;36mHuggingFacePipeline._call\u001b[1;34m(self, prompt, stop, run_manager, **kwargs)\u001b[0m\n\u001b[0;32m    160\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_call\u001b[39m(\n\u001b[0;32m    161\u001b[0m     \u001b[39mself\u001b[39m,\n\u001b[0;32m    162\u001b[0m     prompt: \u001b[39mstr\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    165\u001b[0m     \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs: Any,\n\u001b[0;32m    166\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mstr\u001b[39m:\n\u001b[1;32m--> 167\u001b[0m     response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mpipeline(prompt)\n\u001b[0;32m    168\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpipeline\u001b[39m.\u001b[39mtask \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mtext-generation\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[0;32m    169\u001b[0m         \u001b[39m# Text generation return includes the starter text.\u001b[39;00m\n\u001b[0;32m    170\u001b[0m         text \u001b[39m=\u001b[39m response[\u001b[39m0\u001b[39m][\u001b[39m\"\u001b[39m\u001b[39mgenerated_text\u001b[39m\u001b[39m\"\u001b[39m][\u001b[39mlen\u001b[39m(prompt) :]\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\transformers\\pipelines\\text2text_generation.py:165\u001b[0m, in \u001b[0;36mText2TextGenerationPipeline.__call__\u001b[1;34m(self, *args, **kwargs)\u001b[0m\n\u001b[0;32m    136\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__call__\u001b[39m(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[0;32m    137\u001b[0m \u001b[39m    \u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m    138\u001b[0m \u001b[39m    Generate the output text(s) using text(s) given as inputs.\u001b[39;00m\n\u001b[0;32m    139\u001b[0m \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    162\u001b[0m \u001b[39m          ids of the generated text.\u001b[39;00m\n\u001b[0;32m    163\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 165\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39m\u001b[39m__call__\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[0;32m    166\u001b[0m     \u001b[39mif\u001b[39;00m (\n\u001b[0;32m    167\u001b[0m         \u001b[39misinstance\u001b[39m(args[\u001b[39m0\u001b[39m], \u001b[39mlist\u001b[39m)\n\u001b[0;32m    168\u001b[0m         \u001b[39mand\u001b[39;00m \u001b[39mall\u001b[39m(\u001b[39misinstance\u001b[39m(el, \u001b[39mstr\u001b[39m) \u001b[39mfor\u001b[39;00m el \u001b[39min\u001b[39;00m args[\u001b[39m0\u001b[39m])\n\u001b[0;32m    169\u001b[0m         \u001b[39mand\u001b[39;00m \u001b[39mall\u001b[39m(\u001b[39mlen\u001b[39m(res) \u001b[39m==\u001b[39m \u001b[39m1\u001b[39m \u001b[39mfor\u001b[39;00m res \u001b[39min\u001b[39;00m result)\n\u001b[0;32m    170\u001b[0m     ):\n\u001b[0;32m    171\u001b[0m         \u001b[39mreturn\u001b[39;00m [res[\u001b[39m0\u001b[39m] \u001b[39mfor\u001b[39;00m res \u001b[39min\u001b[39;00m result]\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\transformers\\pipelines\\base.py:1140\u001b[0m, in \u001b[0;36mPipeline.__call__\u001b[1;34m(self, inputs, num_workers, batch_size, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1132\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mnext\u001b[39m(\n\u001b[0;32m   1133\u001b[0m         \u001b[39miter\u001b[39m(\n\u001b[0;32m   1134\u001b[0m             \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mget_iterator(\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   1137\u001b[0m         )\n\u001b[0;32m   1138\u001b[0m     )\n\u001b[0;32m   1139\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m-> 1140\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mrun_single(inputs, preprocess_params, forward_params, postprocess_params)\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\transformers\\pipelines\\base.py:1147\u001b[0m, in \u001b[0;36mPipeline.run_single\u001b[1;34m(self, inputs, preprocess_params, forward_params, postprocess_params)\u001b[0m\n\u001b[0;32m   1145\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrun_single\u001b[39m(\u001b[39mself\u001b[39m, inputs, preprocess_params, forward_params, postprocess_params):\n\u001b[0;32m   1146\u001b[0m     model_inputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpreprocess(inputs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mpreprocess_params)\n\u001b[1;32m-> 1147\u001b[0m     model_outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mforward(model_inputs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mforward_params)\n\u001b[0;32m   1148\u001b[0m     outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mpostprocess(model_outputs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mpostprocess_params)\n\u001b[0;32m   1149\u001b[0m     \u001b[39mreturn\u001b[39;00m outputs\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\transformers\\pipelines\\base.py:1046\u001b[0m, in \u001b[0;36mPipeline.forward\u001b[1;34m(self, model_inputs, **forward_params)\u001b[0m\n\u001b[0;32m   1044\u001b[0m     \u001b[39mwith\u001b[39;00m inference_context():\n\u001b[0;32m   1045\u001b[0m         model_inputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ensure_tensor_on_device(model_inputs, device\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdevice)\n\u001b[1;32m-> 1046\u001b[0m         model_outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_forward(model_inputs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mforward_params)\n\u001b[0;32m   1047\u001b[0m         model_outputs \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ensure_tensor_on_device(model_outputs, device\u001b[39m=\u001b[39mtorch\u001b[39m.\u001b[39mdevice(\u001b[39m\"\u001b[39m\u001b[39mcpu\u001b[39m\u001b[39m\"\u001b[39m))\n\u001b[0;32m   1048\u001b[0m \u001b[39melse\u001b[39;00m:\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\transformers\\pipelines\\text2text_generation.py:187\u001b[0m, in \u001b[0;36mText2TextGenerationPipeline._forward\u001b[1;34m(self, model_inputs, **generate_kwargs)\u001b[0m\n\u001b[0;32m    185\u001b[0m generate_kwargs[\u001b[39m\"\u001b[39m\u001b[39mmax_length\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m generate_kwargs\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mmax_length\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel\u001b[39m.\u001b[39mconfig\u001b[39m.\u001b[39mmax_length)\n\u001b[0;32m    186\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mcheck_inputs(input_length, generate_kwargs[\u001b[39m\"\u001b[39m\u001b[39mmin_length\u001b[39m\u001b[39m\"\u001b[39m], generate_kwargs[\u001b[39m\"\u001b[39m\u001b[39mmax_length\u001b[39m\u001b[39m\"\u001b[39m])\n\u001b[1;32m--> 187\u001b[0m output_ids \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel\u001b[39m.\u001b[39mgenerate(\u001b[39m*\u001b[39m\u001b[39m*\u001b[39mmodel_inputs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mgenerate_kwargs)\n\u001b[0;32m    188\u001b[0m out_b \u001b[39m=\u001b[39m output_ids\u001b[39m.\u001b[39mshape[\u001b[39m0\u001b[39m]\n\u001b[0;32m    189\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mframework \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mpt\u001b[39m\u001b[39m\"\u001b[39m:\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\torch\\utils\\_contextlib.py:115\u001b[0m, in \u001b[0;36mcontext_decorator.<locals>.decorate_context\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    112\u001b[0m \u001b[39m@functools\u001b[39m\u001b[39m.\u001b[39mwraps(func)\n\u001b[0;32m    113\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mdecorate_context\u001b[39m(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs):\n\u001b[0;32m    114\u001b[0m     \u001b[39mwith\u001b[39;00m ctx_factory():\n\u001b[1;32m--> 115\u001b[0m         \u001b[39mreturn\u001b[39;00m func(\u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\transformers\\generation\\utils.py:1602\u001b[0m, in \u001b[0;36mGenerationMixin.generate\u001b[1;34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, **kwargs)\u001b[0m\n\u001b[0;32m   1585\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39massisted_decoding(\n\u001b[0;32m   1586\u001b[0m         input_ids,\n\u001b[0;32m   1587\u001b[0m         assistant_model\u001b[39m=\u001b[39massistant_model,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   1598\u001b[0m         \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mmodel_kwargs,\n\u001b[0;32m   1599\u001b[0m     )\n\u001b[0;32m   1600\u001b[0m \u001b[39mif\u001b[39;00m generation_mode \u001b[39m==\u001b[39m GenerationMode\u001b[39m.\u001b[39mGREEDY_SEARCH:\n\u001b[0;32m   1601\u001b[0m     \u001b[39m# 11. run greedy search\u001b[39;00m\n\u001b[1;32m-> 1602\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mgreedy_search(\n\u001b[0;32m   1603\u001b[0m         input_ids,\n\u001b[0;32m   1604\u001b[0m         logits_processor\u001b[39m=\u001b[39mlogits_processor,\n\u001b[0;32m   1605\u001b[0m         stopping_criteria\u001b[39m=\u001b[39mstopping_criteria,\n\u001b[0;32m   1606\u001b[0m         pad_token_id\u001b[39m=\u001b[39mgeneration_config\u001b[39m.\u001b[39mpad_token_id,\n\u001b[0;32m   1607\u001b[0m         eos_token_id\u001b[39m=\u001b[39mgeneration_config\u001b[39m.\u001b[39meos_token_id,\n\u001b[0;32m   1608\u001b[0m         output_scores\u001b[39m=\u001b[39mgeneration_config\u001b[39m.\u001b[39moutput_scores,\n\u001b[0;32m   1609\u001b[0m         return_dict_in_generate\u001b[39m=\u001b[39mgeneration_config\u001b[39m.\u001b[39mreturn_dict_in_generate,\n\u001b[0;32m   1610\u001b[0m         synced_gpus\u001b[39m=\u001b[39msynced_gpus,\n\u001b[0;32m   1611\u001b[0m         streamer\u001b[39m=\u001b[39mstreamer,\n\u001b[0;32m   1612\u001b[0m         \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mmodel_kwargs,\n\u001b[0;32m   1613\u001b[0m     )\n\u001b[0;32m   1615\u001b[0m \u001b[39melif\u001b[39;00m generation_mode \u001b[39m==\u001b[39m GenerationMode\u001b[39m.\u001b[39mCONTRASTIVE_SEARCH:\n\u001b[0;32m   1616\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m model_kwargs[\u001b[39m\"\u001b[39m\u001b[39muse_cache\u001b[39m\u001b[39m\"\u001b[39m]:\n",
      "File \u001b[1;32mc:\\anaconda3\\envs\\langchain\\lib\\site-packages\\transformers\\generation\\utils.py:2507\u001b[0m, in \u001b[0;36mGenerationMixin.greedy_search\u001b[1;34m(self, input_ids, logits_processor, stopping_criteria, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, streamer, **model_kwargs)\u001b[0m\n\u001b[0;32m   2502\u001b[0m     unfinished_sequences \u001b[39m=\u001b[39m unfinished_sequences\u001b[39m.\u001b[39mmul(\n\u001b[0;32m   2503\u001b[0m         next_tokens\u001b[39m.\u001b[39mtile(eos_token_id_tensor\u001b[39m.\u001b[39mshape[\u001b[39m0\u001b[39m], \u001b[39m1\u001b[39m)\u001b[39m.\u001b[39mne(eos_token_id_tensor\u001b[39m.\u001b[39munsqueeze(\u001b[39m1\u001b[39m))\u001b[39m.\u001b[39mprod(dim\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m)\n\u001b[0;32m   2504\u001b[0m     )\n\u001b[0;32m   2506\u001b[0m     \u001b[39m# stop when each sentence is finished\u001b[39;00m\n\u001b[1;32m-> 2507\u001b[0m     \u001b[39mif\u001b[39;00m unfinished_sequences\u001b[39m.\u001b[39mmax() \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m:\n\u001b[0;32m   2508\u001b[0m         this_peer_finished \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[0;32m   2510\u001b[0m \u001b[39m# stop if we exceed the maximum length\u001b[39;00m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from langchain.llms import HuggingFacePipeline\n",
    "from langchain.chains.summarize import load_summarize_chain\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from transformers import (AutoModelForCausalLM, AutoTokenizer, pipeline)\n",
    "import torch\n",
    "\n",
    "\"\"\"\n",
    "本地LLM加载,使用HuggingFacePipeline连接到langchain\n",
    "\"\"\"\n",
    "localmodels = [\n",
    "                r'E:\\llama\\text-generation-webui\\models\\Baichuan2-7B-Base',\n",
    "                r'E:\\llama\\text-generation-webui\\models\\Baichuan2-7B-Chat',\n",
    "                r'E:\\llama\\text-generation-webui\\models\\THUDM_chatglm-6b'\n",
    "                ]\n",
    "modeid = localmodels[0]\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\n",
    "    modeid, use_fast=False, trust_remote_code=True)\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    modeid, trust_remote_code=True, device_map='cuda:0',torch_dtype=torch.bfloat16)\n",
    "\n",
    "taskid = \"text2text-generation\"\n",
    "\n",
    "pipe = pipeline(\n",
    "    task=taskid,\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    max_length=4096\n",
    "    # device=0\n",
    ")\n",
    "\n",
    "llm = HuggingFacePipeline(pipeline=pipe)\n",
    "with open('state_of_the_union.txt', 'r') as file:\n",
    "    text = file.read() \n",
    "\n",
    "# 打印小说的前285个字符\n",
    "print (text[:285])\n",
    "\n",
    "num_tokens = llm.get_num_tokens(text)\n",
    "\n",
    "print (f\"There are {num_tokens} tokens in your file\") \n",
    "\n",
    "text_splitter = RecursiveCharacterTextSplitter(separators=[\"\\n\\n\", \"\\n\"], chunk_size=5000, chunk_overlap=350)\n",
    "# 虽然我使用的是 RecursiveCharacterTextSplitter，但是你也可以使用其他工具\n",
    "docs = text_splitter.create_documents([text])\n",
    "\n",
    "print (f\"You now have {len(docs)} docs intead of 1 piece of text\")\n",
    "\n",
    "# 设置 lang chain\n",
    "# 使用 map_reduce的chain_type，这样可以将多个文档合并成一个\n",
    "chain = load_summarize_chain(llm=llm, chain_type='map_reduce') # verbose=True 展示运行日志\n",
    "\n",
    "# Use it. This will run through the many documents, summarize the chunks, then get a summary of the summary.\n",
    "# 典型的map reduce的思路去解决问题，将文章拆分成多个部分，再将多个部分分别进行 summarize，最后再进行 合并，对 summarys 进行 summary\n",
    "output = chain.run(docs)\n",
    "print (output)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "langchain",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
